import requests
import time
import Sha1Util
import pymysql
import threading
import time
from concurrent.futures import ThreadPoolExecutor,as_completed,Future
from lxml import etree
import A地区

# 关联到jian表 获取到id,name,href
sql = "select id,name,href from jian "
conn = pymysql.connect(host='localhost', user='root', password='0830', database='moonlightcat_test')
cursor = conn.cursor()

lock = threading.RLock()  # 获取锁

# 定义获取jian表的数据的方法（id，name，href）
def getIDAndname_hrefFromjian():
    cursor.execute(sql)
    result = dict(cursor.fetchall())
    return result

def getXiangqingye():
    result = A地区.getDatafromjian()  # id,name,href
    for data in result:
        name = data[1]
        href = data[2]
        headers = {
            "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/95.0.4638.54 Safari/537.36 Edg/95.0.1020.40"
        }
        resp = requests.get(url=href, headers=headers)
        resp.encoding = "utf-8"
        xiangqingye_text = resp.text
        leixing_dict = {
            "title": "",
            "gongzuoleixing": "",
            "zhaopinrenshu": "",
            "shangbandidian": "",
            "shijianyaoqiu": "",
            "gongzhong": "",
            "meizhouzhishao": "",
            "shangbanshiduan": "",
            "jiesuanfangshi": "",
            "gongzi": "",
            "gs_title": "",
            "gs_jieshao": "",
            "gs_dizhi": ""
        }
        tree = etree.HTML(xiangqingye_text)
        title = [i.strip() for i in tree.xpath('//div[@class="job_header"]/h1/text()')]  # 工作名称
        leixing_dict['title'] = title[0] if title else ""
        gongzuoleixing = [i.strip() for i in tree.xpath('//div[@class="job_base"]/a/text()')]  # 工作类型
        leixing_dict['gongzuoleixing'] = gongzuoleixing[0] if gongzuoleixing else ""
        # 解析招聘人数等信息
        li_people = tree.xpath('//div[@class="job_content"]//li')
        for t in li_people:
            tit = [i.strip() for i in t.xpath('./span[@class="tit"]/text()')]
            con = [i.strip() for i in t.xpath('./span[@class="con"]/text()')]
            if tit and con:
                leixing_dict['zhaopinrenshu'] = con[0] if '招聘人数' in tit[0] else leixing_dict['zhaopinrenshu']
                leixing_dict['shangbandidian'] = con[0] if '上班地点' in tit[0] else leixing_dict[
                    'shangbandidian']
                leixing_dict['shijianyaoqiu'] = con[0] if '时间要求' in tit[0] else leixing_dict['shijianyaoqiu']
                leixing_dict['gongzhong'] = con[0] if '工作种类' in tit[0] else leixing_dict['gongzhong']
                leixing_dict['meizhouzhishao'] = con[0] if '每周至少' in tit[0] else leixing_dict[
                    'meizhouzhishao']
                leixing_dict['shangbanshiduan'] = con[0] if '上班时段' in tit[0] else leixing_dict[
                    'shangbanshiduan']
                leixing_dict['jiesuanfangshi'] = con[0] if '结算方式' in tit[0] else leixing_dict[
                    'jiesuanfangshi']
                leixing_dict['gongzi'] = con[0] if '基本工资' in tit[0] else leixing_dict['gongzi']
        #     # 解析公司信息
        company = tree.xpath('//div[@class="company_info"]')
        if company:
            gs_title = [i.strip() for i in company[0].xpath('./a/text()')]
            gs_jieshao = [i.strip() for i in company[0].xpath('./p[1]/text()')]
            gs_dizhi = [i.strip() for i in company[0].xpath('./p[2]/text()')]
            leixing_dict['gs_title'] = gs_title[0] if gs_title else leixing_dict['gs_title']
            leixing_dict['gs_jieshao'] = gs_jieshao[0] if gs_jieshao else leixing_dict[
                'gs_jieshao']  # /html/body/section/article/div/div[1]/div[3]/div[2]/div[1]/div[2]/p[1]/span
            leixing_dict['gs_dizhi'] = gs_dizhi[0] if gs_dizhi else leixing_dict['gs_dizhi']
        work_xiangqing = [i.strip() for i in tree.xpath('//*[@id="job_detail"]//text() ')]
        id = Sha1Util.jia_mi(leixing_dict["title"] + href + name)
        list = ""
        for a in range(0, len(work_xiangqing)):
            if work_xiangqing[a] != "":
                list = list + work_xiangqing[a]
        sql = """insert  into xiangqing(id,工作名称,链接,工作类型,招聘人数,上班地点,时间要求,工作种类,每周至少,
        上班时段,结算方式,基本工资,工作详情,公司名称,公司介绍,公司地址)
            values('%s','%s','%s','%s','%s','%s','%s','%s','%s','%s','%s','%s','%s','%s','%s','%s')
                        """ % (id, leixing_dict["title"], href, leixing_dict["gongzuoleixing"],
                               leixing_dict["zhaopinrenshu"], leixing_dict["shangbandidian"],
                               leixing_dict["shijianyaoqiu"], leixing_dict["gongzhong"],
                               leixing_dict["meizhouzhishao"], leixing_dict["shangbanshiduan"],
                               leixing_dict["jiesuanfangshi"], leixing_dict["gongzi"],
                               list, leixing_dict["gs_title"], leixing_dict["gs_jieshao"],
                               leixing_dict["gs_dizhi"])
        try:
            lock.acquire()  # 加锁
            cursor.execute(sql)
            conn.commit()
            print("数据插入成功", name+sql)
            lock.release()  # 解锁
        except Exception as e:
            # 事务回滚
            conn.rollback()
            print(e)
time.sleep(0.05)

def duoxiancheng():
     with ThreadPoolExecutor(14) as th:
         th.submit(getXiangqingye())
     time.sleep(0.05)

if __name__ == '__main__':
    duoxiancheng()
